#coding=UTF-8
#! /usr/local/bin/python
# -*- coding: UTF-8 -*-
# Filename=new_backup.py

'''
Created on 2012-10-16

@author: Dujx
'''
from datetime import date,timedelta
import os
import shutil
import time
import ConfigParser
import commands
import hashlib
import traceback
import sys
import signal

def showMessage(str):
        '''打印字符串'''
        print  str.decode('utf-8')

def makeMd5(fp): 
    m = hashlib.md5() 
    f = open(fp, 'rb') 
    buffer = 8192         
    while 1: 
        chunk = f.read(buffer) 
        if not chunk : break
        m.update(chunk)          
    f.close() 
    return m.hexdigest()

def copyFile(sp,dp):
    flag=False
    if os.path.exists(os.path.dirname(sp)):
        try:
            if not os.path.exists(os.path.dirname(dp)):
	        os.makedirs(os.path.dirname(dp))            
	    if not (os.path.isfile(sp)):
                #showMessage("源文件"+sp+"不存在")
                flag=False
            try:
               shutil.copy(sp,dp)
            except TimedOutExc:
                print "Time out"
            flag=True
        except:
            flag=False
    else:
        flag=False
    return flag
def try_int(s):
    #"Convert to integer if possible."
    try: return int(s)
    except: return s

def natsort_key(s):
    #"Used internally to get a tuple by which s is sorted."
    import re
    return map(try_int, re.findall(r'(\d+|\D+)', s))
def natcmp(a, b):
    #"Natural string comparison, case sensitive."
    return cmp(natsort_key(a), natsort_key(b))

def filelist(dirname):
    flist=[]
    if not (os.path.exists(dirname)):
        showMessage(dirname+"不存在")
        return False;
    filenames=os.listdir(dirname)
    if len(filenames) == 0:
       #showMessage("empty folder："+dirname;
       return [];
    else:
        for name in filenames:
            if (name[-4:]==".log" and name[:5]!="error"):
                flist.append(name[:-4])
        # sort the list                
        flist.sort(natcmp)
        return flist
def logFile(efile,estr):    
    f=open(os.path.join(efile,time.strftime('%Y-%m-%d')),'ab')
    f.write("["+time.strftime('%Y-%m-%d %X')+"]`"+estr.replace('\n', '')+'  \n')
    f.close()

cf=ConfigParser.ConfigParser()
cf.read("D:/Temp/tongbu/new/ucpacker.ini")

sp=cf.get('common','sp');
dp=cf.get('common','dp');
logsp=cf.get('common','logsp');
logdp=cf.get('common','logdp');
daydiff = int(cf.get('common','daydiff'))
ecount = int(cf.get('common','ecount'))

#时间跨度startday-today
today = date.today();
startday = today - timedelta(daydiff-1);

#建立错误日志夹
logErr = os.path.join(logdp,"error")
if not (os.path.exists(logErr)):
    os.makedirs(logErr)
#建立运行状态日志夹
logStat = os.path.join(logdp,"status")
if not (os.path.exists(logStat)):
    os.makedirs(logStat)
try:
    #if commands.getstatusoutput('ps aux|grep /home/ucun/htdocs/python/down/script/new_backup.py|grep -v grep|wc -l')==(0,'2'):
        # 计时
        st = time.time();
        # 总计数
        gcount = 0
        # 成功计数
        scount = 0
        # 失败计数
        fcount = 0
        showMessage("[当前时间] "+time.strftime('%Y-%m-%d %X'))
        # 从startday到today循环扫描
        for i in range(0,daydiff):
            checkday = startday + timedelta(i)
            sdir = os.path.join(logsp,str(checkday))        
            ddir = os.path.join(logdp,str(checkday))
            showMessage("-----------------------[开始扫描] "+str(checkday)+"的log文件 --------------------------")
            #找到ddir最新的log       
            if not (os.path.exists(ddir)):
                if (os.path.exists(sdir)):
                    #showMessage("建立文件夹"+ddir;
                    os.makedirs(ddir) 
                else:
                    showMessage("[扫描结果] 没有文件需要同步")
                    continue;
                
            #差集文件列表，初始为空
            difflist=[]
            # 当前下载目录文件列表
            downlist = filelist(ddir)       
            #showMessage("[扫描结果] 下载目录当前文件名列表："+str(downlist)
            
            # 如果不为空,MD5比对最后一个log
            if not (downlist == []):
                newest = downlist[-1]
                showMessage("[扫描结果] 需要比对: "+newest+".log")
                if(makeMd5(os.path.join(sdir,newest+".log")) != makeMd5(os.path.join(ddir,newest+".log"))):
                    showMessage("[比对结果] "+newest+".log的md5不同")
                    #差集加上最后一个log
                    difflist.append(newest);                
                else:
                    showMessage("[比对结果] "+newest+".log的md5相同")
                            
            # 当前打包服务器目录文件列表
            uplist = filelist(sdir)
            #showMessage("[扫描结果] 打包目录当前文件名列表："+str(uplist)      
            # 扩展差集
            difflist.extend(list(set(uplist) - set(downlist)))
            
            if(len(difflist) == 0):
                showMessage("[扫描结果] 不需要同步")            
            else: 
                # 自然排序
                difflist.sort(natcmp)    
                
                # 循环读取差集文件的内容         
                for log in difflist:
                    #一次循环总计数
                    gsum=0                                        
                    #打包服务器的log                
                    slog = os.path.join(sdir,log+".log")
                    showMessage("[准备同步] "+"读取打包服务器文件"+str(slog))
                    #list形式读取打包服务器要拉取的log内容
                    sm = time.time()
                    if not (os.path.exists(slog)):
                        showMessage("[出错] 打包服务器无此文件: "+slog)
                        logFile(logErr,"打包服务器无此文件: "+slog)
                        continue;
                    fp = open(slog,"rb");                       
                    uplines = fp.readlines()                
                    fp.close()
                    showMessage("[准备同步] 读取打包log耗时："+str(time.time()-sm)+"s")
                    #下载服务器相对应的log

                    
                    dlog = os.path.join(ddir,log+".log");
                    
                    if not (os.path.exists(dlog)):
                        #如果log不存在，则从第一行开始拷贝
                        s=0;
                    else:
                        #否则从最后一行续传
                        s = sum(1 for line in open(dlog))
                        
                    #showMessage(uplines[s];
                    #源log总行数
                    count = len(uplines);
                    need = count-s
                    if(need > 0):
                        showMessage("[准备同步] 需要同步"+str(need)+"个包")
                    else:
                        emsg = "[出错] 需要同步"+str(need)+"个包"
                        showMessage(emsg)
                        logFile(logErr,emsg+" `"+dlog)
                        continue
                    
                    num = 0;
                    writeStr ="";
                    showMessage("[正在同步] 请耐心等待...")
                    sync_str ="[正在同步]".decode('utf-8')
                    percent =0.1
                    milestone = int(float(need)* percent)
                    fails = False
                    for i in range (s,count):
                        # 复制文件                        
                        #showMessage("[正在同步] 同步第"+str(i+1)+"个包")
                        fn = uplines[i].rstrip('\n');                    
                        sfile = sp+"down"+fn
                        dfile = dp+"down"+fn
                        #exestime = time.time()
                        result = False                                                
                        result = copyFile(sfile,dfile)
                        #time.sleep(3)
                        
                        #记录文件(不管成功与否)
                        writeStr += uplines[i];
                        num += 1;
                        gsum += 1;
                        gcount += 1
                        sys.stdout.write('\r%s %s' % (sync_str,str(gsum)))
                        sys.stdout.flush()      
                                           
                        if(num >= ecount or i == (count-1)):
                            #记录文件                           
                            f = open(os.path.join(ddir,log+".log"),"ab")                        
                            f.write("%s" % writeStr)              
                            f.close()
                            writeStr="";
                            num = 0
                        
                        #如果同步失败   
                        if(result == False):
                            fcount +=1
                            fails = True
                            #记录error.log
                            showMessage("\r\n[复制失败] "+fn)
                            fr = open(os.path.join(ddir,"error-"+log+".log"),"ab")
                            fr.write("%s\n" % fn)
                            fr.close()
                        else:
                            scount +=1
                    showMessage("\n[同步结果] 完成"+str(slog)+"同步")
                    
        stat = "[同步耗时]" +str(time.time()-st)+"s,[总包数]"+str(gcount)+",[成功数]"+str(scount) +",[失败数]"+str(fcount)
        showMessage("----------------------- "+stat+" -------------------------\n")
        logFile(logStat,stat)
except Exception,err:
    showMessage("[发生异常] 请查看日志\n"+str(err))
    logFile(logErr,traceback.format_exc())
  



    
    
    
    
    
    
    
    
    
    
    
    
    
